import requests
from lxml import etree
import tkinter as tk
from tkinter import ttk, messagebox, filedialog
import threading
import time
from functools import wraps


class WebContentExtractor:
    def __init__(self, root):
        self.root = root
        self.root.title("网页内容抓取工具")
        self.root.geometry("650x600")

        # 创建主框架
        self.main_frame = ttk.Frame(root, padding="10")
        self.main_frame.pack(fill=tk.BOTH, expand=True)

        # URL输入部分
        ttk.Label(self.main_frame, text="目标URL:").grid(row=0, column=0, sticky=tk.W)
        self.url_entry = ttk.Entry(self.main_frame, width=70)
        self.url_entry.grid(row=0, column=1, sticky=tk.W, pady=5)
        self.url_entry.insert(0, "请输入爬取目标网址")

        # XPath输入部分
        ttk.Label(self.main_frame, text="目录XPath:").grid(row=1, column=0, sticky=tk.W)
        self.catalogue_xpath_entry = ttk.Entry(self.main_frame, width=70)
        self.catalogue_xpath_entry.grid(row=1, column=1, sticky=tk.W, pady=5)
        self.catalogue_xpath_entry.insert(0, 'XPath表达式(例如：//div[@class="panel-body"]/p/text())')

        ttk.Label(self.main_frame, text="标题XPath:").grid(row=2, column=0, sticky=tk.W)
        self.title_xpath_entry = ttk.Entry(self.main_frame, width=70)
        self.title_xpath_entry.grid(row=2, column=1, sticky=tk.W, pady=5)
        self.title_xpath_entry.insert(0, 'XPath表达式(例如：//div[@class="panel-body"]/p/text())')

        ttk.Label(self.main_frame, text="内容XPath:").grid(row=3, column=0, sticky=tk.W)
        self.content_xpath_entry = ttk.Entry(self.main_frame, width=70)
        self.content_xpath_entry.grid(row=3, column=1, sticky=tk.W, pady=5)
        self.content_xpath_entry.insert(0, 'XPath表达式(例如：//div[@class="panel-body"]/p/text())')

        # 广告处理设置
        ttk.Label(self.main_frame, text="跳过开头行数:").grid(row=4, column=0, sticky=tk.W)
        self.skip_start_entry = ttk.Spinbox(self.main_frame, from_=0, to=20, width=5)
        self.skip_start_entry.grid(row=4, column=1, sticky=tk.W, pady=5)
        self.skip_start_entry.set(2)

        ttk.Label(self.main_frame, text="跳过结尾行数:").grid(row=5, column=0, sticky=tk.W)
        self.skip_end_entry = ttk.Spinbox(self.main_frame, from_=0, to=20, width=5)
        self.skip_end_entry.grid(row=5, column=1, sticky=tk.W, pady=5)
        self.skip_end_entry.set(1)

        # 重试机制设置
        ttk.Label(self.main_frame, text="最大重试次数:").grid(row=6, column=0, sticky=tk.W)
        self.retry_entry = ttk.Spinbox(self.main_frame, from_=1, to=10, width=5)
        self.retry_entry.grid(row=6, column=1, sticky=tk.W, pady=5)
        self.retry_entry.set(3)

        ttk.Label(self.main_frame, text="重试间隔(秒):").grid(row=7, column=0, sticky=tk.W)
        self.retry_interval_entry = ttk.Spinbox(self.main_frame, from_=1, to=10, width=5)
        self.retry_interval_entry.grid(row=7, column=1, sticky=tk.W, pady=5)
        self.retry_interval_entry.set(2)

        # 输出文件名
        ttk.Label(self.main_frame, text="输出文件名:").grid(row=8, column=0, sticky=tk.W)
        self.filename_entry = ttk.Entry(self.main_frame, width=70)
        self.filename_entry.grid(row=8, column=1, sticky=tk.W, pady=5)
        self.filename_entry.insert(0, "")

        # 浏览按钮
        self.browse_button = ttk.Button(self.main_frame, text="浏览...", command=self.browse_file)
        self.browse_button.grid(row=8, column=2, sticky=tk.W, padx=5)

        # 进度条
        self.progress = ttk.Progressbar(self.main_frame, orient=tk.HORIZONTAL, length=500, mode='determinate')
        self.progress.grid(row=9, column=0, columnspan=3, pady=10)

        # 日志输出
        ttk.Label(self.main_frame, text="操作日志:").grid(row=10, column=0, sticky=tk.W)
        self.log_text = tk.Text(self.main_frame, height=12, width=85)
        self.log_text.grid(row=11, column=0, columnspan=3, sticky=tk.W)

        # 按钮框架
        self.button_frame = ttk.Frame(self.main_frame)
        self.button_frame.grid(row=12, column=0, columnspan=3, pady=10)

        self.start_button = ttk.Button(self.button_frame, text="开始抓取", command=self.start_extraction)
        self.start_button.pack(side=tk.LEFT, padx=5)

        self.stop_button = ttk.Button(self.button_frame, text="停止", command=self.stop_extraction, state=tk.DISABLED)
        self.stop_button.pack(side=tk.LEFT, padx=5)

        # 状态变量
        self.is_running = False
        self.stop_requested = False

        # 默认请求头
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
        }

    def browse_file(self):
        filename = filedialog.asksaveasfilename(
            initialfile=self.filename_entry.get(),
            defaultextension=".txt",
            filetypes=[("Text files", "*.txt"), ("All files", "*.*")]
        )
        if filename:
            self.filename_entry.delete(0, tk.END)
            self.filename_entry.insert(0, filename)

    def log(self, message):
        self.log_text.insert(tk.END, message + "\n")
        self.log_text.see(tk.END)
        self.root.update()

    def retry_request(func):
        @wraps(func)
        def wrapper(self, *args, **kwargs):
            max_retries = int(self.retry_entry.get())
            retry_interval = int(self.retry_interval_entry.get())

            for attempt in range(1, max_retries + 1):
                try:
                    return func(self, *args, **kwargs)
                except (requests.exceptions.RequestException, etree.XMLSyntaxError) as e:
                    if attempt == max_retries:
                        raise
                    error_msg = f"请求失败 (尝试 {attempt}/{max_retries}): {str(e)}"
                    self.log(error_msg)
                    if self.stop_requested:
                        self.log("用户请求停止，取消重试")
                        raise
                    time.sleep(retry_interval)
        return wrapper

    @retry_request
    def safe_get_request(self, url):
        response = requests.get(url, headers=self.headers, timeout=60)
        response.raise_for_status()
        response.encoding = "utf-8"
        return response.content.decode()

    def start_extraction(self):
        if self.is_running:
            return

        # 获取用户输入（url）
        url = self.url_entry.get().strip()
        # 获取用户输入（Xpath）
        catalogue_xpath = self.catalogue_xpath_entry.get().strip()
        title_xpath = self.title_xpath_entry.get().strip()
        content_xpath = self.content_xpath_entry.get().strip()
        skip_start = int(self.skip_start_entry.get())
        skip_end = int(self.skip_end_entry.get())
        filename = self.filename_entry.get().strip()

        if not url or not catalogue_xpath or not title_xpath or not content_xpath or not filename:
            messagebox.showerror("错误", "请填写所有必填字段！")
            return

        # 清空日志
        self.log_text.delete(1.0, tk.END)

        # 更新UI状态
        self.is_running = True
        self.stop_requested = False
        self.start_button.config(state=tk.DISABLED)
        self.stop_button.config(state=tk.NORMAL)
        self.progress["value"] = 0

        # 在新线程中执行抓取
        thread = threading.Thread(
            target=self.extract_content,
            args=(url, catalogue_xpath, title_xpath, content_xpath, skip_start, skip_end, filename),
            daemon=True
        )
        thread.start()

    def stop_extraction(self):
        self.stop_requested = True
        self.log("正在停止抓取...")

    def extract_content(self, url, catalogue_xpath, title_xpath, content_xpath, skip_start, skip_end, filename):
        try:
            self.log(f"开始抓取: {url}")

            # 1. 获取目录页（带重试）
            self.log("正在获取目录页...")
            try:
                html = self.safe_get_request(url)
            except Exception as e:
                self.log(f"目录页获取失败: {str(e)}")
                return

            # 解析目录页
            tree = etree.HTML(html)
            content_urls = tree.xpath(catalogue_xpath)

            if not content_urls:
                self.log("错误: 未找到任何内容链接，请检查XPath表达式！")
                return

            total_pages = len(content_urls)
            self.log(f"找到 {total_pages} 个内容页面")

            # 2. 遍历每个内容页
            with open(filename, "w", encoding="utf-8") as file:
                for i, url in enumerate(content_urls, 1):
                    if self.stop_requested:
                        self.log("用户请求停止抓取")
                        break

                    # 更新进度
                    progress = (i / total_pages) * 100
                    self.progress["value"] = progress
                    self.root.update()

                    self.log(f"正在处理第 {i}/{total_pages} 页: {url}")

                    try:
                        # 获取正文页（带重试）
                        try:
                            content_html = self.safe_get_request(url)
                        except Exception as e:
                            self.log(f"页面获取失败，跳过: {str(e)}")
                            continue

                        # 解析正文页
                        tree = etree.HTML(content_html)

                        # 获取标题
                        title = tree.xpath(title_xpath)
                        if not title:
                            self.log(f"警告: 第 {i} 页未找到标题，跳过")
                            continue

                        # 获取内容
                        content = tree.xpath(content_xpath)
                        if not content:
                            self.log(f"警告: 第 {i} 页未找到内容，跳过")
                            continue

                        # 写入文件
                        file.write(title[0] + '\n')

                        # 处理广告内容（跳过开头和结尾指定行数）
                        start = skip_start
                        end = len(content) - skip_end if skip_end > 0 else len(content)

                        for line in content[start:end]:
                            file.write(line.strip() + "\n")

                        file.write("\n")

                    except Exception as e:
                        self.log(f"错误: 处理第 {i} 页时出错 - {str(e)}")
                        continue

            self.log("抓取完成！")

        except Exception as e:
            self.log(f"发生错误: {str(e)}")
        finally:
            # 恢复UI状态
            self.is_running = False
            self.stop_requested = False
            self.start_button.config(state=tk.NORMAL)
            self.stop_button.config(state=tk.DISABLED)
            self.progress["value"] = 100
            self.root.update()


if __name__ == "__main__":
    root = tk.Tk()
    app = WebContentExtractor(root)
    root.mainloop()